import scrapy

from apps.tax_honor.tax_honor.base_clean_spider import BaseCleanMixin
from apps.tax_honor.tax_honor.items import NetTaxHonorResponseItem
from utils.tools import urlencode, parse_url_params, del_redundant_blank_character


class ZjHzzjxxhjTaxHonorSpider(scrapy.Spider, BaseCleanMixin):
    name = "zj_kxjjt_honor"
    url = "https://kjt.zj.gov.cn/col/col1228971387/index.html"
    province = "浙江省"
    city = ""
    county = ""
    hierarchy = "省级"

    def start_requests(self):
        datas = [
            dict(
                url="https://kjt.zj.gov.cn/module/jpage/dataproxy.jsp",
                params={"startrecord": "1", "endrecord": "20", "perpage": "15"},
                data={
                    "col": "1",
                    "appid": "1",
                    "webid": "3387",
                    "path": "/",
                    "columnid": "1228971387",
                    "sourceContentType": "3",
                    "unitid": "5491660",
                    "webname": "浙江省科学技术厅",
                    "permissiontype": "0",
                },
                cb_kwargs={
                    "city": "",
                    "county": "",
                    "hierarchy": "省级",
                },
            ),
            dict(
                url="https://fzggw.zj.gov.cn/module/jpage/dataproxy.jsp",
                params={"startrecord": "1", "endrecord": "45", "perpage": "15"},
                data={
                    "col": "1",
                    "appid": "1",
                    "webid": "3185",
                    "path": "/",
                    "columnid": "1599544",
                    "sourceContentType": "1",
                    "unitid": "4892867",
                    "webname": "浙江省发展和改革委员会",
                    "permissiontype": "0",
                },
                cb_kwargs={
                    "city": "",
                    "county": "",
                    "hierarchy": "省级",
                },
            ),
            dict(
                url="https://jyt.zj.gov.cn/module/jpage/dataproxy.jsp",
                params={"startrecord": "1", "endrecord": "45", "perpage": "15"},
                data={
                    "col": "1",
                    "appid": "1",
                    "webid": "3099",
                    "path": "/",
                    "columnid": "1229266336",
                    "sourceContentType": "1",
                    "unitid": "6258453",
                    "webname": "浙江省教育厅",
                    "permissiontype": "0",
                },
                cb_kwargs={
                    "city": "",
                    "county": "",
                    "hierarchy": "省级",
                },
            ),
            dict(
                url="https://mzt.zj.gov.cn/module/jpage/dataproxy.jsp",
                params={"startrecord": "1", "endrecord": "45", "perpage": "15"},
                data={
                    "col": "1",
                    "appid": "1",
                    "webid": "3206",
                    "path": "/",
                    "columnid": "1632726",
                    "sourceContentType": "1",
                    "unitid": "5312084",
                    "webname": "浙江省民政厅",
                    "permissiontype": "0",
                },
                cb_kwargs={
                    "city": "",
                    "county": "",
                    "hierarchy": "省级",
                },
            ),
            dict(
                url="https://rlsbt.zj.gov.cn/module/jpage/dataproxy.jsp",
                params={"startrecord": "1", "endrecord": "30", "perpage": "10"},
                data={
                    "col": "1",
                    "appid": "1",
                    "webid": "2758",
                    "path": "/",
                    "columnid": "1229101517",
                    "sourceContentType": "1",
                    "unitid": "7382343",
                    "webname": "浙江省人力资源和社会保障厅",
                    "permissiontype": "0",
                },
                cb_kwargs={
                    "city": "",
                    "county": "",
                    "hierarchy": "省级",
                },
            ),
            dict(
                url="http://sthjt.zj.gov.cn/module/jpage/dataproxy.jsp",
                params={"startrecord": "1", "endrecord": "60", "perpage": "20"},
                data={
                    "col": "1",
                    "appid": "1",
                    "webid": "1756",
                    "path": "/",
                    "columnid": "1229263559",
                    "sourceContentType": "1",
                    "unitid": "6246852",
                    "webname": "浙江省生态环境厅",
                    "permissiontype": "0",
                },
                cb_kwargs={
                    "city": "",
                    "county": "",
                    "hierarchy": "省级",
                },
            ),
            dict(
                url="https://nynct.zj.gov.cn/module/jpage/morecolumndataproxy.jsp",
                params={"startrecord": "1", "endrecord": "45", "perpage": "15"},
                data={
                    "col": "1",
                    "appid": "1",
                    "webid": "2801",
                    "path": "/",
                    "columnid": "1589297,1229235418",
                    "b_vir": "0",
                    "virinfos": "",
                    "sourceContentType": "3",
                    "unitid": "7512149",
                    "keyWordCount": "999",
                    "webname": "浙江省农业农村厅",
                },
                cb_kwargs={
                    "city": "",
                    "county": "",
                    "hierarchy": "省级",
                },
            ),
            dict(
                url="https://zcom.zj.gov.cn/module/jpage/dataproxy.jsp",
                params={"startrecord": "1", "endrecord": "10", "perpage": "10"},
                data={
                    "col": "1",
                    "appid": "1",
                    "webid": "2757",
                    "path": "/",
                    "columnid": "1384587",
                    "sourceContentType": "1",
                    "unitid": "4450235",
                    "webname": "浙江省商务厅",
                    "permissiontype": "0",
                },
                cb_kwargs={
                    "city": "",
                    "county": "",
                    "hierarchy": "省级",
                },
            ),
            dict(
                url="https://ct.zj.gov.cn/module/jpage/dataproxy.jsp",
                params={"startrecord": "1", "endrecord": "45", "perpage": "15"},
                data={
                    "col": "1",
                    "appid": "1",
                    "webid": "3214",
                    "path": "/",
                    "columnid": "1229678763",
                    "sourceContentType": "1",
                    "unitid": "7814811",
                    "webname": "浙江省文化广电和旅游厅",
                    "permissiontype": "0",
                },
                cb_kwargs={
                    "city": "",
                    "county": "",
                    "hierarchy": "省级",
                },
            ),
        ]

        for x in datas:
            yield scrapy.FormRequest(
                method="POST",
                url=x["url"] + "?" + urlencode(x["params"]),
                formdata=x["data"],
                callback=self.parse,
                dont_filter=True,
                cb_kwargs=x["cb_kwargs"],
            )

    def parse(self, response, **kwargs):
        city = kwargs["city"]
        county = kwargs["county"]
        hierarchy = kwargs["hierarchy"]
        root_url, params = parse_url_params(response.request.url)
        _, data = parse_url_params(response.request.body.decode())
        page = kwargs.get("page")
        if not page:
            total_record = int(response.xpath("//datastore//totalrecord/text()").get())
            for i in range(46, total_record + 1, 45):
                yield scrapy.FormRequest(
                    method="POST",
                    url=root_url + "?" + urlencode({**params, "startrecord": str(i), "endrecord": str(i + 44)}),
                    formdata=data,
                    callback=self.parse,
                    dont_filter=True,
                    cb_kwargs={"page": i, **kwargs},
                )

        response = response.replace(body=response.text.replace("<![CDATA[", "").replace("]]>", "").encode("utf-8"))
        for li in response.xpath("//datastore//recordset//record"):
            if not li.xpath(".//a/@href").get():
                li = scrapy.Selector(text=li.xpath(".").get())
            url = response.urljoin(li.xpath(".//a/@href").get())
            document_title = li.xpath(".//a/@title").get() or li.xpath(".//a/text()").get()
            publish_date = li.xpath(".//span/text()").get() or li.xpath(".//*[contains(@class, 'time')]/text()").get() or li.xpath(".").re_first(r"\d{4}-\d{2}-\d{2}")
            if document_title:
                if self.check_utils.check_title_is_valid(document_title):
                    yield scrapy.Request(
                        url=url,
                        callback=self.parse_detail,
                        cb_kwargs={
                            "document_title": document_title,
                            "publish_date": publish_date,
                            "document_url": url,
                            "city": city,
                            "county": county,
                            "hierarchy": hierarchy,
                        },
                    )

    def parse_detail(self, response, **kwargs):
        city = kwargs["city"]
        county = kwargs["county"]
        hierarchy = kwargs["hierarchy"]
        document_title = kwargs.get("document_title")
        publish_date = kwargs.get("publish_date")
        decare_department = response.xpath("string(.)").re_first(r"发布机构\s*[:：]\s*([^\s*]{1,50})")
        document_url = kwargs.get("document_url")
        hrefs = response.xpath("//a/@href").getall()
        attachment_url = []
        if hrefs:
            for href in hrefs:
                if href.endswith((".pdf", ".docx", ".doc", ".xlsx", ".xls", ".zip", ".et")):
                    href = response.urljoin(href)
                    attachment_url.append(href)
        response_item = {
            "province": self.province,
            "city": city,
            "county": county,
            "document_title": document_title,
            "publish_date": publish_date,
            "decare_department": decare_department,
            "hierarchy": hierarchy,
            "document_url": document_url,
            "attachment_url": attachment_url,
            "deadline": None,
            "source": self.name,
            "item_count": 0,
            "response": del_redundant_blank_character(response.body.decode()),
        }
        yield NetTaxHonorResponseItem(**response_item)
        if response_item["attachment_url"]:
            for url in response_item["attachment_url"]:
                yield scrapy.Request(
                    url=url,
                    callback=self.parse_spider_detail,
                    cb_kwargs={"response_item": response_item},
                )
        yield scrapy.Request(
            url=response_item["document_url"],
            callback=self.parse_spider_detail,
            cb_kwargs={"response_item": response_item},
        )


if __name__ == "__main__":
    from scrapy import cmdline, Selector

    cmdline.execute(argv=["scrapy", "crawl", "zj_kxjjt_honor"])
